{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/backup/Research/HOI_plus/lib/utils/env.py:44: UserWarning: \n",
      "This call to matplotlib.use() has no effect because the backend has already\n",
      "been chosen; matplotlib.use() must be called *before* pylab, matplotlib.pyplot,\n",
      "or matplotlib.backends is imported for the first time.\n",
      "\n",
      "The backend was *originally* set to 'module://ipykernel.pylab.backend_inline' by the following code:\n",
      "  File \"/root/anaconda3/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n",
      "    \"__main__\", mod_spec)\n",
      "  File \"/root/anaconda3/lib/python3.6/runpy.py\", line 85, in _run_code\n",
      "    exec(code, run_globals)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py\", line 16, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n",
      "    app.start()\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/ipykernel/kernelapp.py\", line 478, in start\n",
      "    self.io_loop.start()\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n",
      "    super(ZMQIOLoop, self).start()\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py\", line 888, in start\n",
      "    handler_func(fd_obj, events)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n",
      "    return fn(*args, **kwargs)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n",
      "    self._handle_recv()\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n",
      "    self._run_callback(callback, msg)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n",
      "    callback(*args, **kwargs)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n",
      "    return fn(*args, **kwargs)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n",
      "    return self.dispatch_shell(stream, msg)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 233, in dispatch_shell\n",
      "    handler(stream, idents, msg)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n",
      "    user_expressions, allow_stdin)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/ipykernel/ipkernel.py\", line 208, in do_execute\n",
      "    res = shell.run_cell(code, store_history=store_history, silent=silent)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/ipykernel/zmqshell.py\", line 537, in run_cell\n",
      "    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2728, in run_cell\n",
      "    interactivity=interactivity, compiler=compiler, result=result)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2850, in run_ast_nodes\n",
      "    if self.run_code(code, result):\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2910, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-1-c588a418a201>\", line 6, in <module>\n",
      "    get_ipython().run_line_magic('matplotlib', 'inline')\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2095, in run_line_magic\n",
      "    result = fn(*args,**kwargs)\n",
      "  File \"<decorator-gen-107>\", line 2, in matplotlib\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/magic.py\", line 187, in <lambda>\n",
      "    call = lambda f, *a, **k: f(*a, **k)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/magics/pylab.py\", line 99, in matplotlib\n",
      "    gui, backend = self.shell.enable_matplotlib(args.gui)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2978, in enable_matplotlib\n",
      "    pt.activate_matplotlib(backend)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/pylabtools.py\", line 308, in activate_matplotlib\n",
      "    matplotlib.pyplot.switch_backend(backend)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/matplotlib/pyplot.py\", line 232, in switch_backend\n",
      "    matplotlib.use(newbackend, warn=False, force=True)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/matplotlib/__init__.py\", line 1305, in use\n",
      "    reload(sys.modules['matplotlib.backends'])\n",
      "  File \"/root/anaconda3/lib/python3.6/importlib/__init__.py\", line 166, in reload\n",
      "    _bootstrap._exec(spec, module)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/matplotlib/backends/__init__.py\", line 14, in <module>\n",
      "    line for line in traceback.format_stack()\n",
      "\n",
      "\n",
      "  matplotlib.use('Agg')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/root/backup/Research/HOI_plus/tools ..coco evaling\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/backup/Research/HOI_plus/lib/utils/vis.py:36: UserWarning: \n",
      "This call to matplotlib.use() has no effect because the backend has already\n",
      "been chosen; matplotlib.use() must be called *before* pylab, matplotlib.pyplot,\n",
      "or matplotlib.backends is imported for the first time.\n",
      "\n",
      "The backend was *originally* set to 'module://ipykernel.pylab.backend_inline' by the following code:\n",
      "  File \"/root/anaconda3/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n",
      "    \"__main__\", mod_spec)\n",
      "  File \"/root/anaconda3/lib/python3.6/runpy.py\", line 85, in _run_code\n",
      "    exec(code, run_globals)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py\", line 16, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n",
      "    app.start()\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/ipykernel/kernelapp.py\", line 478, in start\n",
      "    self.io_loop.start()\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n",
      "    super(ZMQIOLoop, self).start()\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py\", line 888, in start\n",
      "    handler_func(fd_obj, events)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n",
      "    return fn(*args, **kwargs)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n",
      "    self._handle_recv()\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n",
      "    self._run_callback(callback, msg)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n",
      "    callback(*args, **kwargs)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n",
      "    return fn(*args, **kwargs)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n",
      "    return self.dispatch_shell(stream, msg)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 233, in dispatch_shell\n",
      "    handler(stream, idents, msg)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n",
      "    user_expressions, allow_stdin)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/ipykernel/ipkernel.py\", line 208, in do_execute\n",
      "    res = shell.run_cell(code, store_history=store_history, silent=silent)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/ipykernel/zmqshell.py\", line 537, in run_cell\n",
      "    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2728, in run_cell\n",
      "    interactivity=interactivity, compiler=compiler, result=result)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2850, in run_ast_nodes\n",
      "    if self.run_code(code, result):\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2910, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-1-c588a418a201>\", line 6, in <module>\n",
      "    get_ipython().run_line_magic('matplotlib', 'inline')\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2095, in run_line_magic\n",
      "    result = fn(*args,**kwargs)\n",
      "  File \"<decorator-gen-107>\", line 2, in matplotlib\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/magic.py\", line 187, in <lambda>\n",
      "    call = lambda f, *a, **k: f(*a, **k)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/magics/pylab.py\", line 99, in matplotlib\n",
      "    gui, backend = self.shell.enable_matplotlib(args.gui)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2978, in enable_matplotlib\n",
      "    pt.activate_matplotlib(backend)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/IPython/core/pylabtools.py\", line 308, in activate_matplotlib\n",
      "    matplotlib.pyplot.switch_backend(backend)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/matplotlib/pyplot.py\", line 232, in switch_backend\n",
      "    matplotlib.use(newbackend, warn=False, force=True)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/matplotlib/__init__.py\", line 1305, in use\n",
      "    reload(sys.modules['matplotlib.backends'])\n",
      "  File \"/root/anaconda3/lib/python3.6/importlib/__init__.py\", line 166, in reload\n",
      "    _bootstrap._exec(spec, module)\n",
      "  File \"/root/anaconda3/lib/python3.6/site-packages/matplotlib/backends/__init__.py\", line 14, in <module>\n",
      "    line for line in traceback.format_stack()\n",
      "\n",
      "\n",
      "  matplotlib.use('Agg')\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import sys\n",
    "%matplotlib inline\n",
    "# import _init_paths\n",
    "from PIL import Image\n",
    "import matplotlib.pyplot as plt\n",
    "# from ult.config import cfg\n",
    "import pickle\n",
    "import json\n",
    "import numpy as np\n",
    "import cv2\n",
    "import os\n",
    "import ipdb\n",
    "import torch\n",
    "import scipy.misc as miscp\n",
    "import _init_paths\n",
    "from core.config import cfg, merge_cfg_from_file, merge_cfg_from_list, assert_and_infer_cfg\n",
    "from core.test_engine import run_inference, get_inference_dataset, extend_results\n",
    "from datasets.json_dataset import JsonDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "original image numer: 4946\n",
      "original annotation number: 42999\n",
      "Done (t=1.50s)\n",
      "creating index...\n",
      "index created!\n",
      "final image number: 4946\n",
      "final anns number: 42999\n",
      "loading vcoco annotations...\n",
      "add precomp box from /root/backup/Research/HOI_plus/data/cache/vcoco_test_precomp_boxes_ican.json\n"
     ]
    }
   ],
   "source": [
    "cfg.TEST.DATASETS = ('vcoco_test',)\n",
    "cfg.MODEL.NUM_CLASSES = 81\n",
    "cfg.TEST.PRECOMPUTED_PROPOSALS = False\n",
    "cfg.MODEL.VCOCO_ON = True\n",
    "cfg.VCOCO.USE_PRECOMP_BOX = True\n",
    "dataset_name, proposal_file = get_inference_dataset(0, is_parent=False)\n",
    "json_dataset = JsonDataset(dataset_name)\n",
    "vcocodb = json_dataset.get_roidb(gt=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '../Outputs/e2e_interact_net_R-50-FPN_1x/Dec10-15-24-10_wanbo_node23_step/test_0.5_0.4_all/hoi_vcoco_test_results.pkl'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-998c4b5455fe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mdet_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'../Outputs/e2e_interact_net_R-50-FPN_1x/Dec10-15-24-10_wanbo_node23_step/test_0.5_0.4_all/hoi_vcoco_test_results.pkl'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mall_hoi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdet_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'latin1'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;31m# image_ids = list(all_hoi.keys())\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '../Outputs/e2e_interact_net_R-50-FPN_1x/Dec10-15-24-10_wanbo_node23_step/test_0.5_0.4_all/hoi_vcoco_test_results.pkl'"
     ]
    }
   ],
   "source": [
    "det_name = '../Outputs/e2e_interact_net_R-50-FPN_1x/Dec10-15-24-10_wanbo_node23_step/test_0.5_0.4_all/hoi_vcoco_test_results.pkl'\n",
    "all_hoi = pickle.load(open(det_name, 'rb') , encoding='latin1')\n",
    "# image_ids = list(all_hoi.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------Reporting Role AP (%)------------------\n",
      "               hold-obj: AP = 38.80 (#pos = 3608)\n",
      "              sit-instr: AP = 24.93 (#pos = 1916)\n",
      "             ride-instr: AP = 64.06 (#pos = 556)\n",
      "               look-obj: AP = 36.09 (#pos = 3347)\n",
      "              hit-instr: AP = 71.91 (#pos = 349)\n",
      "                hit-obj: AP = 45.02 (#pos = 349)\n",
      "                eat-obj: AP = 38.99 (#pos = 521)\n",
      "              eat-instr: AP = 4.13 (#pos = 521)\n",
      "             jump-instr: AP = 47.85 (#pos = 635)\n",
      "              lay-instr: AP = 23.80 (#pos = 387)\n",
      "    talk_on_phone-instr: AP = 52.28 (#pos = 285)\n",
      "              carry-obj: AP = 34.45 (#pos = 472)\n",
      "              throw-obj: AP = 44.35 (#pos = 244)\n",
      "              catch-obj: AP = 46.01 (#pos = 246)\n",
      "              cut-instr: AP = 34.01 (#pos = 269)\n",
      "                cut-obj: AP = 37.70 (#pos = 269)\n",
      " work_on_computer-instr: AP = 58.67 (#pos = 410)\n",
      "              ski-instr: AP = 46.58 (#pos = 424)\n",
      "             surf-instr: AP = 78.60 (#pos = 486)\n",
      "       skateboard-instr: AP = 82.04 (#pos = 417)\n",
      "            drink-instr: AP = 32.99 (#pos = 82)\n",
      "               kick-obj: AP = 66.61 (#pos = 180)\n",
      "               read-obj: AP = 23.56 (#pos = 111)\n",
      "        snowboard-instr: AP = 67.91 (#pos = 277)\n",
      "Average Role [scenario_1] AP = 45.89\n",
      "---------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "json_dataset.VCOCO._do_role_eval(vcocodb, all_hoi, ovr_thresh=0.5, eval_type='scenario_1', plot_save_path=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_overlap(boxes, ref_box):\n",
    "    ixmin = np.maximum(boxes[:, 0], ref_box[0])\n",
    "    iymin = np.maximum(boxes[:, 1], ref_box[1])\n",
    "    ixmax = np.minimum(boxes[:, 2], ref_box[2])\n",
    "    iymax = np.minimum(boxes[:, 3], ref_box[3])\n",
    "    # maximum zero\n",
    "    iw = np.maximum(ixmax - ixmin + 1., 0.)\n",
    "    ih = np.maximum(iymax - iymin + 1., 0.)\n",
    "    inters = iw * ih\n",
    "\n",
    "    # union\n",
    "    uni = ((ref_box[2] - ref_box[0] + 1.) * (ref_box[3] - ref_box[1] + 1.) +\n",
    "           (boxes[:, 2] - boxes[:, 0] + 1.) *\n",
    "           (boxes[:, 3] - boxes[:, 1] + 1.) - inters)\n",
    "\n",
    "    overlaps = inters / uni\n",
    "    return overlaps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "46368 36549 62905 9755 47879 201 9029\n",
      "212688\n"
     ]
    }
   ],
   "source": [
    "count = 0\n",
    "num_actions = json_dataset.VCOCO.num_actions\n",
    "roles = json_dataset.VCOCO.roles\n",
    "fp1 = fp2 = fp3 = fp4 = fp5 = fp6 = fp7 = tp = 0\n",
    "all_res = []\n",
    "set_zero_ret = {}\n",
    "\n",
    "for i in range(len(vcocodb)):\n",
    "    res = []\n",
    "    image_id = vcocodb[i]['id']\n",
    "\n",
    "    pred_roles = all_hoi[image_id]['roles']\n",
    "    pred_agents = all_hoi[image_id]['agents']\n",
    "    gt_human_inds, gt_act_inds, gt_r_ids = np.where(vcocodb[i]['gt_role_id'] > -1)\n",
    "    gt_obj_inds = vcocodb[i]['gt_role_id'][gt_human_inds, gt_act_inds, gt_r_ids]\n",
    "    if len(gt_obj_inds) == 0:\n",
    "#         set_zero_ret[image_id] = {'agents': pred_agents, 'roles': pred_roles}\n",
    "        continue\n",
    "    gt_all_relboxes = vcocodb[i]['boxes'][np.unique(gt_obj_inds)]\n",
    "        \n",
    "    gt_h_inds = np.where(vcocodb[i]['gt_classes'] == 1)[0]\n",
    "    gt_h_boxes = vcocodb[i]['boxes'][gt_h_inds]\n",
    "    gt_actions = vcocodb[i]['gt_actions'][gt_h_inds]\n",
    "    ignore = np.any(gt_actions == -1, axis=1)\n",
    "    \n",
    "    for aid in range(num_actions):\n",
    "        if len(roles[aid]) < 2:\n",
    "            continue\n",
    "\n",
    "        for rid in range(len(roles[aid]) - 1):\n",
    "            \n",
    "            covered = np.zeros((gt_h_boxes.shape[0]), dtype=np.bool)\n",
    "            gt_role_inds = vcocodb[i]['gt_role_id'][gt_h_inds, aid, rid]\n",
    "            gt_roles = -np.ones_like(gt_h_boxes)\n",
    "            for j in range(gt_h_boxes.shape[0]):\n",
    "                if gt_role_inds[j] > -1:\n",
    "                    gt_roles[j] = vcocodb[i]['boxes'][gt_role_inds[j]]\n",
    "            \n",
    "            agent_boxes = pred_agents[:, :4]\n",
    "            role_boxes = pred_roles[:, 5 * aid: 5 * aid + 4, rid]\n",
    "            agent_scores = pred_roles[:, 5 * aid + 4, rid]\n",
    "            \n",
    "            if role_boxes.shape[0] == 0: \n",
    "                continue\n",
    "                \n",
    "            valid = np.where(np.isnan(agent_scores) == False)[0]\n",
    "            agent_scores = agent_scores[valid]\n",
    "            agent_boxes = agent_boxes[valid, :]\n",
    "            role_boxes = role_boxes[valid, :]\n",
    "            \n",
    "            idx = agent_scores.argsort()[::-1]\n",
    "            for j in idx:  # in this image, this action with highest action score\n",
    "                pred_box = agent_boxes[j, :]\n",
    "                overlaps = get_overlap(gt_h_boxes, pred_box)  # gt_h_boxes: gt human box\n",
    "\n",
    "                jmax = overlaps.argmax()  # which gt_box best matches this detected box\n",
    "                ovmax = overlaps.max()\n",
    "\n",
    "                if ignore[jmax]:\n",
    "                    continue\n",
    "                count += 1\n",
    "                \n",
    "                ov_all_relboxes = get_overlap(gt_all_relboxes, role_boxes[j, :])\n",
    "                ov_all_boxes = get_overlap(vcocodb[i]['boxes'], role_boxes[j, :])\n",
    "                ov_max_all_relboxes = ov_all_relboxes.max()\n",
    "                ov_max_all_boxes = ov_all_boxes.max()\n",
    "\n",
    "                gt_role_j_human = vcocodb[i]['gt_role_id'][gt_h_inds[jmax]]\n",
    "                gt_box_j_objs = vcocodb[i]['boxes'][np.unique(gt_role_j_human[np.where(gt_role_j_human > -1)])]\n",
    "                if len(gt_box_j_objs) == 0:\n",
    "                    ov_max_j_objs = -1\n",
    "                else:\n",
    "                    ov_j_objs = get_overlap(gt_box_j_objs, role_boxes[j, :])\n",
    "                    ov_max_j_objs = ov_j_objs.max()\n",
    "                    \n",
    "                if np.all(gt_roles[jmax, :] == -1):\n",
    "                    if np.all(role_boxes[j, :] == 0.0) or np.all(np.isnan(role_boxes[j, :])):\n",
    "                        ov_role = 1.0\n",
    "                    else:\n",
    "                        ov_role = -1.0\n",
    "                else:\n",
    "                    ov_role = get_overlap(gt_roles[jmax, :].reshape((1, 4)), role_boxes[j, :])\n",
    "                    \n",
    "                is_true_action = (gt_actions[jmax, aid] == 1)\n",
    "                    \n",
    "                if ovmax < 0.5: # person mis-loc\n",
    "                    fp1 += 1\n",
    "                    res.append(np.concatenate([pred_box, role_boxes[j, :], [aid, rid, agent_scores[j]], [1]]))\n",
    "                elif 0 <= ov_max_all_boxes < 0.5:\n",
    "                    if is_true_action == True and ov_max_j_objs == -1.0: # occlusion\n",
    "                        fp6 += 1\n",
    "                        res.append(np.concatenate([pred_box, role_boxes[j, :], [aid, rid, agent_scores[j]], [6]]))\n",
    "                    else: # obj misloc\n",
    "                        fp2 += 1\n",
    "                        res.append(np.concatenate([pred_box, role_boxes[j, :], [aid, rid, agent_scores[j]], [2]]))\n",
    "                elif 0 <= ov_max_all_relboxes < 0.5: # misgrouping, obj not attend any rel\n",
    "                    fp3 += 1\n",
    "                    \n",
    "#                     pred_roles[valid[j], 5 * aid + 4, rid] = 0\n",
    "                    \n",
    "                    res.append(np.concatenate([pred_box, role_boxes[j, :], [aid, rid, agent_scores[j]], [3]]))\n",
    "                elif ov_max_j_objs < 0.5: # misgrouping, obj attend rel but not such person\n",
    "                    fp4 += 1\n",
    "                    \n",
    "#                     res.append(np.concatenate([pred_box, role_boxes[j, :], [aid, rid, agent_scores[j]], [4]]))\n",
    "                    \n",
    "                    pred_roles[valid[j], 5 * aid + 4, rid] = 0\n",
    "                elif ov_role < 0.5: # prediction error\n",
    "                    fp5 += 1\n",
    "#                     pred_roles[valid[j], 5 * aid + 4, rid] = 0\n",
    "                    res.append(np.concatenate([pred_box, role_boxes[j, :], [aid, rid, agent_scores[j]], [5]]))\n",
    "                elif (ovmax >= 0.5) & (ov_role >= 0.5):  # true positive\n",
    "                    if not covered[jmax]:\n",
    "                        tp += 1\n",
    "                        res.append(np.concatenate([pred_box, role_boxes[j, :], [aid, rid, agent_scores[j]], [0]]))\n",
    "                        covered[jmax] = True\n",
    "                    else:   # triplet duplicate\n",
    "                        fp7 += 1\n",
    "                        res.append(np.concatenate([pred_box, role_boxes[j, :], [aid, rid, agent_scores[j]], [7]]))\n",
    "#     set_zero_ret[image_id] = {'agents': pred_agents, 'roles':pred_roles}\n",
    "    ret = {}\n",
    "    ret['im_id'] = image_id\n",
    "    ret['gt_classes'] = vcocodb[i]['gt_classes']\n",
    "    ret['gt_boxes'] = vcocodb[i]['boxes']\n",
    "    ret['gt_actions'] = vcocodb[i]['gt_actions']\n",
    "    ret['gt_role_inds'] = vcocodb[i]['gt_role_id']\n",
    "    ret['pred_triplets'] = np.array(res)\n",
    "    all_res.append(ret)\n",
    "print(fp1,fp2,fp3,fp4,fp5,fp6,tp)\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "46368 36549 62905 9755 47879 201\n",
      "gt triplet num: 12423\n",
      "tp num: 9029\n",
      "fp num: 203657\n",
      "recall: 0.7267970699508975\n",
      "person misloc: 0.2276769273828054\n",
      "obj misloc: 0.17946350972468414\n",
      "misgrouping, obj not attend any rel: 0.3088771807499865\n",
      "misgrouping, obj attend rel but not such person: 0.04789916379009806\n",
      "prediction wrong: 0.23509626479816553\n",
      "occlusion: 0.0009869535542603495\n"
     ]
    }
   ],
   "source": [
    "print(fp1,fp2,fp3,fp4,fp5,fp6)\n",
    "fps = fp1+fp2+fp3+fp4+fp5+fp6\n",
    "gt_num = 12423\n",
    "print('gt triplet num:', gt_num)\n",
    "print('tp num:', tp)\n",
    "print('fp num:', fps)\n",
    "print('recall:', tp/gt_num)\n",
    "print('person misloc:', fp1/fps)\n",
    "print('obj misloc:', fp2/fps)\n",
    "print('misgrouping, obj not attend any rel:', fp3/fps)\n",
    "print('misgrouping, obj attend rel but not such person:', fp4/fps)\n",
    "print('prediction wrong:', fp5/fps)\n",
    "print('occlusion:', fp6/fps)\n",
    "# print('duplicate:', fp7/fps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------Reporting Role AP (%)------------------\n",
      "               hold-obj: AP = 46.02 (#pos = 3608)\n",
      "              sit-instr: AP = 28.86 (#pos = 1916)\n",
      "             ride-instr: AP = 68.40 (#pos = 556)\n",
      "               look-obj: AP = 49.11 (#pos = 3347)\n",
      "              hit-instr: AP = 77.01 (#pos = 349)\n",
      "                hit-obj: AP = 52.69 (#pos = 349)\n",
      "                eat-obj: AP = 44.36 (#pos = 521)\n",
      "              eat-instr: AP = 6.05 (#pos = 521)\n",
      "             jump-instr: AP = 53.36 (#pos = 635)\n",
      "              lay-instr: AP = 30.53 (#pos = 387)\n",
      "    talk_on_phone-instr: AP = 55.40 (#pos = 285)\n",
      "              carry-obj: AP = 43.54 (#pos = 472)\n",
      "              throw-obj: AP = 68.80 (#pos = 244)\n",
      "              catch-obj: AP = 61.72 (#pos = 246)\n",
      "              cut-instr: AP = 38.24 (#pos = 269)\n",
      "                cut-obj: AP = 44.30 (#pos = 269)\n",
      " work_on_computer-instr: AP = 64.79 (#pos = 410)\n",
      "              ski-instr: AP = 52.10 (#pos = 424)\n",
      "             surf-instr: AP = 81.12 (#pos = 486)\n",
      "       skateboard-instr: AP = 84.80 (#pos = 417)\n",
      "            drink-instr: AP = 45.71 (#pos = 82)\n",
      "               kick-obj: AP = 82.30 (#pos = 180)\n",
      "               read-obj: AP = 29.65 (#pos = 111)\n",
      "        snowboard-instr: AP = 74.11 (#pos = 277)\n",
      "Average Role [scenario_1] AP = 53.46\n",
      "---------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "json_dataset.VCOCO._do_role_eval(vcocodb, set_zero_ret, ovr_thresh=0.5, eval_type='scenario_1', plot_save_path=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# file = './Outputs/e2e_interact_net_R-50-FPN_1x/Nov14-09-23-26_wanbo_node33_step/test/hoi_vcoco_triplet_results.pkl'\n",
    "# save_file = './Outputs/e2e_interact_net_R-50-FPN_1x/Nov14-09-23-26_wanbo_node33_step/images/'\n",
    "file = '../Outputs/e2e_interact_net_R-50-FPN_1x/Dec10-15-24-10_wanbo_node23_step/test/hoi_vcoco_triplet_results_ican.pkl'\n",
    "save_file = '../Outputs/e2e_interact_net_R-50-FPN_1x/Dec10-15-24-10_wanbo_node23_step/images/'\n",
    "# file = './ican/hoi_vcoco_triplet_results_ican.pkl'\n",
    "# save_file = './ican/images/'\n",
    "if not os.path.exists(save_file):\n",
    "    os.mkdir(save_file)\n",
    "f = open(file, 'rb')\n",
    "all_res = pickle.load(f, encoding='latin1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "no_gt_image_num = 0\n",
    "pred_no_human_num = 0\n",
    "fp1 = fp2 = fp3 = fp4 = fp5 = fp6 = tp = 0\n",
    "gt_num = 0\n",
    "count = 0\n",
    "for i, ele in enumerate(all_res):\n",
    "    image_id = ele['im_id']\n",
    "    gt_classes = ele['gt_classes']\n",
    "    gt_boxes = ele['gt_boxes']\n",
    "    gt_actions = ele['gt_actions']\n",
    "    pred_triplets = ele['pred_triplets']\n",
    "    gt_role_inds = ele['gt_role_inds']\n",
    "\n",
    "    gt_human_inds, gt_act_inds, gt_r_ids = np.where(gt_role_inds>-1)\n",
    "    gt_obj_inds = gt_role_inds[gt_human_inds, gt_act_inds, gt_r_ids]\n",
    "    gt_human_box = gt_boxes[gt_human_inds].astype(np.int)\n",
    "    gt_obj_boxes = gt_boxes[gt_obj_inds].astype(np.int)\n",
    "    \n",
    "    gt_num += len(gt_human_inds)\n",
    "    if len(gt_human_inds) == 0:\n",
    "        no_gt_image_num += 1\n",
    "        continue\n",
    "    if pred_triplets.shape[0] == 0:\n",
    "        pred_no_human_num += 1\n",
    "        continue\n",
    "    \n",
    "    \n",
    "    scores = pred_triplets[:, -2]\n",
    "    idx = scores.argsort()[::-1]\n",
    "#     triplets = pred_triplets[idx[:5]]\n",
    "#     sorted_score = scores[idx[:5]]\n",
    "    triplets = pred_triplets[idx]\n",
    "    sorted_score = scores[idx]\n",
    "\n",
    "    for m, t in enumerate(triplets):\n",
    "#         if sorted_score[m] < 0.1:\n",
    "#             continue\n",
    "        h_box = tuple(t[:4].astype(np.int))\n",
    "        o_box = tuple(t[4:8].astype(np.int))\n",
    "        aid, rid, score, error_type = t[-4:]\n",
    "        text = action_names[aid.astype(np.int)] + '_' + role_names[rid.astype(np.int)]\n",
    "        if error_type == 0: # tp\n",
    "            tp += 1\n",
    "        elif error_type == 1: # person misloc\n",
    "            fp1 += 1\n",
    "        elif error_type == 2:  # human loc is right but obj misloc\n",
    "            fp2 += 1\n",
    "        elif error_type == 3: # the target object is fg but not in target triplet; misgrouping\n",
    "            fp3 += 1\n",
    "        elif error_type == 4: # person and obj loc is right but predict action wrong; prediction wrong\n",
    "            fp4 += 1\n",
    "        elif error_type == 5: # occlusion\n",
    "            fp5 += 1\n",
    "        elif error_type == 6: # triplet duplicate\n",
    "            fp6 += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4726 13431 25522 3157 32455\n",
      "gt triplet num: 12423\n",
      "tp num: 8951\n",
      "fp num: 79291\n",
      "recall: 0.7205183933027449\n",
      "person misloc: 0.05960323365829665\n",
      "obj misloc: 0.16938870741950537\n",
      "misgrouping: 0.32187764058972645\n",
      "prediction wrong: 0.03981536366044065\n",
      "occlusion: 0.4093150546720309\n"
     ]
    }
   ],
   "source": [
    "print(fp1,fp2,fp3,fp4,fp5)\n",
    "fps = fp1+fp2+fp3+fp4+fp5\n",
    "print('gt triplet num:', gt_num)\n",
    "print('tp num:', tp)\n",
    "print('fp num:', fps)\n",
    "print('recall:', tp/gt_num)\n",
    "print('person misloc:', fp1/fps)\n",
    "print('obj misloc:', fp2/fps)\n",
    "print('misgrouping:', fp3/fps)\n",
    "print('prediction wrong:', fp4/fps)\n",
    "print('occlusion:', fp5/fps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vcoco_anns = '../data/coco/vcoco/vcoco/vcoco_test.json'\n",
    "with open(vcoco_anns, 'r') as f:\n",
    "    vsrl_data = json.load(f)\n",
    "action_names = []\n",
    "for j in vsrl_data:\n",
    "    action_names.append(j['action_name'])\n",
    "\n",
    "role_names = ['obj','instr']\n",
    "red = (255, 0, 0)\n",
    "green = (0, 255, 0)\n",
    "blue = (0, 0, 255)\n",
    "yellow = (255, 255, 0)\n",
    "cyan = (0, 255, 255)\n",
    "# font = cv2.FONT_HERSHEY_SIMPLEX\n",
    "font = cv2.FONT_HERSHEY_PLAIN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:34: DeprecationWarning: `imread` is deprecated!\n",
      "`imread` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``imageio.imread`` instead.\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:41: DeprecationWarning: `imsave` is deprecated!\n",
      "`imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``imageio.imwrite`` instead.\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:51: DeprecationWarning: `imread` is deprecated!\n",
      "`imread` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``imageio.imread`` instead.\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:129: DeprecationWarning: `imsave` is deprecated!\n",
      "`imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``imageio.imwrite`` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50\n",
      "100\n",
      "150\n",
      "200\n",
      "250\n",
      "300\n",
      "350\n",
      "400\n",
      "450\n",
      "500\n",
      "550\n",
      "600\n",
      "650\n",
      "700\n",
      "750\n",
      "800\n",
      "850\n",
      "900\n",
      "950\n",
      "1000\n",
      "1050\n",
      "1100\n",
      "1150\n",
      "1200\n",
      "1250\n",
      "1300\n",
      "1350\n",
      "1400\n",
      "1450\n",
      "1500\n",
      "1550\n",
      "1600\n",
      "1650\n",
      "1700\n",
      "1750\n",
      "1800\n",
      "1850\n",
      "1900\n",
      "1950\n",
      "2000\n",
      "2050\n",
      "2100\n",
      "2150\n",
      "2200\n",
      "2250\n",
      "2300\n",
      "2350\n",
      "2400\n",
      "2450\n",
      "2500\n",
      "2550\n",
      "2600\n",
      "2650\n",
      "2700\n",
      "2750\n",
      "2800\n",
      "2850\n",
      "2900\n",
      "2950\n",
      "3000\n",
      "3050\n",
      "3100\n",
      "3150\n",
      "3200\n",
      "3250\n",
      "3300\n",
      "3350\n",
      "3400\n",
      "3450\n",
      "3500\n",
      "3550\n",
      "3600\n",
      "3650\n",
      "3700\n",
      "3750\n",
      "3800\n",
      "3850\n",
      "3900\n",
      "3950\n",
      "4000\n",
      "4050\n",
      "4100\n",
      "4150\n",
      "4200\n",
      "4250\n",
      "4300\n",
      "4350\n",
      "4400\n",
      "4450\n",
      "4500\n"
     ]
    }
   ],
   "source": [
    "no_gt_image_num = 0\n",
    "pred_no_human_num = 0\n",
    "fp1 = fp2 = fp3 = fp4 = fp5 = fp6 = tp = 0\n",
    "save_file = '../Outputs/e2e_interact_net_R-50-FPN_1x/Dec10-15-24-10_wanbo_node23_step/images1/'\n",
    "for i, ele in enumerate(all_res):\n",
    "    if i%50==0:\n",
    "        print(i)\n",
    "    image_id = ele['im_id']\n",
    "    gt_classes = ele['gt_classes']\n",
    "    gt_boxes = ele['gt_boxes']\n",
    "    gt_actions = ele['gt_actions']\n",
    "    pred_triplets = ele['pred_triplets']\n",
    "    gt_role_inds = ele['gt_role_inds']\n",
    "\n",
    "\n",
    "    gt_human_inds, gt_act_inds, gt_r_ids = np.where(gt_role_inds>-1)\n",
    "    gt_obj_inds = gt_role_inds[gt_human_inds, gt_act_inds, gt_r_ids]\n",
    "    gt_human_box = gt_boxes[gt_human_inds].astype(np.int)\n",
    "    gt_obj_boxes = gt_boxes[gt_obj_inds].astype(np.int)\n",
    "    \n",
    "    if len(gt_human_inds) == 0:\n",
    "        no_gt_image_num += 1\n",
    "        continue\n",
    "    if pred_triplets.shape[0] == 0:\n",
    "        pred_no_human_num += 1\n",
    "        continue\n",
    "        \n",
    "    im_file = '../data/coco/images/val2014/COCO_val2014_' + (str(image_id)).zfill(12) + '.jpg'\n",
    "    save_dir = os.path.join(save_file, 'COCO_val2014_' + (str(image_id)).zfill(12))\n",
    "    if not os.path.exists(save_dir):\n",
    "        os.mkdir(save_dir)\n",
    "        \n",
    "    for n in range(len(gt_human_inds)):\n",
    "        img = miscp.imread(im_file)\n",
    "        gt_h_box = tuple(gt_human_box[n])\n",
    "        gt_o_box = tuple(gt_obj_boxes[n])\n",
    "        text = action_names[gt_act_inds[n]] + '_' + role_names[gt_r_ids[n]]\n",
    "        cv2.putText(img, text, gt_o_box[:2], font, 1.2, green, thickness=2)\n",
    "        cv2.rectangle(img, gt_h_box[:2], gt_h_box[2:], green, 2)\n",
    "        cv2.rectangle(img, gt_o_box[:2], gt_o_box[2:], green, 2)\n",
    "        miscp.imsave(os.path.join(save_dir, 'gt_{}.jpg'.format(n)), img)\n",
    "    \n",
    "    scores = pred_triplets[:, -2]\n",
    "    idx = scores.argsort()[::-1]\n",
    "    triplets = pred_triplets[idx[:10]]\n",
    "    sorted_score = scores[idx[:10]]\n",
    "\n",
    "    for m, t in enumerate(triplets):\n",
    "        if sorted_score[m] < 0.1:\n",
    "            continue\n",
    "        img = miscp.imread(im_file)\n",
    "        h_box = tuple(t[:4].astype(np.int))\n",
    "        o_box = tuple(t[4:8].astype(np.int))\n",
    "        aid, rid, score, error_type = t[-4:]\n",
    "#         ipdb.set_trace()\n",
    "        score_text = \"%.3f\"%(sorted_score[m])\n",
    "        text = action_names[aid.astype(np.int)] + '_' + role_names[rid.astype(np.int)]\n",
    "        if error_type == 0: # tp\n",
    "            tp += 1\n",
    "            h_color = green\n",
    "            o_color = green\n",
    "            a_color = green\n",
    "            cv2.rectangle(img, h_box[:2], h_box[2:], h_color, 2)\n",
    "            cv2.rectangle(img, o_box[:2], o_box[2:], o_color, 2)\n",
    "            cv2.putText(img, text, o_box[:2], font, 1.2, a_color, thickness=2)\n",
    "            cv2.putText(img, score_text, (20, 20), font, 1.2, green, thickness=2)\n",
    "        elif error_type == 1: # person misloc\n",
    "            fp1 += 1\n",
    "            continue\n",
    "            h_color = red\n",
    "            cv2.rectangle(img, h_box[:2], h_box[2:], h_color, 2)\n",
    "            cv2.putText(img, score_text, (20, 20), font, 1.2, green, thickness=2)\n",
    "        elif error_type == 2: # human loc is right but obj misloc\n",
    "            fp2 += 1\n",
    "            continue\n",
    "            h_color = green\n",
    "            o_color = red\n",
    "            a_color = cyan\n",
    "            cv2.rectangle(img, h_box[:2], h_box[2:], h_color, 2)\n",
    "            cv2.rectangle(img, o_box[:2], o_box[2:], o_color, 2)\n",
    "            cv2.putText(img, text, o_box[:2], font, 1.2, a_color, thickness=2)\n",
    "            cv2.putText(img, score_text, (20, 20), font, 1.2, green, thickness=2)\n",
    "        elif error_type == 3:  # the target object is fg but not in target triplet; misgrouping\n",
    "            fp3 += 1\n",
    "            continue\n",
    "            h_color = blue\n",
    "            o_color = blue\n",
    "            a_color = cyan\n",
    "            cv2.rectangle(img, h_box[:2], h_box[2:], h_color, 2)\n",
    "            cv2.rectangle(img, o_box[:2], o_box[2:], o_color, 2)\n",
    "            cv2.putText(img, text, (h_box[0], h_box[1]+20), font, 1.2, a_color, thickness=2)\n",
    "            cv2.putText(img, score_text, (20, 20), font, 1.2, green, thickness=2)\n",
    "        elif error_type == 4:  # the target object is fg but not in target triplet; misgrouping\n",
    "            fp4 += 1\n",
    "            continue\n",
    "            h_color = yellow\n",
    "            o_color = yellow\n",
    "            a_color = cyan\n",
    "            cv2.rectangle(img, h_box[:2], h_box[2:], h_color, 2)\n",
    "            cv2.rectangle(img, o_box[:2], o_box[2:], o_color, 2)\n",
    "            cv2.putText(img, text, (h_box[0], h_box[1]+20), font, 1.2, a_color, thickness=2)\n",
    "            cv2.putText(img, score_text, (20, 20), font, 1.2, green, thickness=2)\n",
    "        elif error_type == 5:  # person and obj loc is right but predict action wrong; prediction wrong\n",
    "            fp5 += 1\n",
    "            h_color = green\n",
    "            o_color = green\n",
    "            a_color = red\n",
    "            cv2.rectangle(img, h_box[:2], h_box[2:], h_color, 2)\n",
    "            cv2.rectangle(img, o_box[:2], o_box[2:], o_color, 2)\n",
    "            cv2.putText(img, text, (h_box[0], h_box[1]+20), font, 1.2, a_color, thickness=2)\n",
    "            cv2.putText(img, score_text, (20, 20), font, 1.2, green, thickness=2)\n",
    "        elif error_type == 6: # occlusion\n",
    "            fp6 += 1\n",
    "            continue\n",
    "            h_color = yellow\n",
    "            a_color = yellow\n",
    "            o_color = yellow\n",
    "            cv2.rectangle(img, h_box[:2], h_box[2:], h_color, 2)\n",
    "            cv2.rectangle(img, o_box[:2], o_box[2:], o_color, 2)\n",
    "            cv2.putText(img, text, (h_box[0], h_box[1]+20), font, 1.2, a_color, thickness=2)\n",
    "            cv2.putText(img, score_text, (20, 20), font, 1.2, green, thickness=2)\n",
    "        elif error_type == 7: # triplet duplicate\n",
    "            fp7 += 1\n",
    "            continue\n",
    "#             h_color = cyan\n",
    "#             o_color = cyan\n",
    "#             cv2.rectangle(img, h_box[:2], h_box[2:], h_color, 2)\n",
    "#             cv2.rectangle(img, o_box[:2], o_box[2:], o_color, 2)\n",
    "        miscp.imsave(os.path.join(save_dir, str(m)+'.jpg'), img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 24\n"
     ]
    }
   ],
   "source": [
    "print(no_gt_image_num, pred_no_human_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2402\n"
     ]
    }
   ],
   "source": [
    "target_path = '../Outputs/e2e_interact_net_R-50-FPN_1x/Dec10-15-24-10_wanbo_node23_step/images_filter/'\n",
    "dirs = os.listdir(save_file)\n",
    "count = 0\n",
    "for diri in dirs:\n",
    "    child = os.path.join('%s/%s' % (save_file, diri))\n",
    "    flag = 0\n",
    "    path_dir1 = os.listdir(child)\n",
    "    for img_name in path_dir1:\n",
    "        if 'gt' not in img_name:\n",
    "            flag += 1\n",
    "    if flag > 0:\n",
    "        os.system('cp -r {} {}'.format(child, target_path))\n",
    "        count += 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------Reporting Role AP (%)------------------\n",
      "               hold-obj: AP = 38.46 (#pos = 3608)\n",
      "              sit-instr: AP = 24.80 (#pos = 1916)\n",
      "             ride-instr: AP = 63.56 (#pos = 556)\n",
      "               look-obj: AP = 36.00 (#pos = 3347)\n",
      "              hit-instr: AP = 70.87 (#pos = 349)\n",
      "                hit-obj: AP = 44.54 (#pos = 349)\n",
      "                eat-obj: AP = 39.03 (#pos = 521)\n",
      "              eat-instr: AP = 4.14 (#pos = 521)\n",
      "             jump-instr: AP = 47.76 (#pos = 635)\n",
      "              lay-instr: AP = 23.86 (#pos = 387)\n",
      "    talk_on_phone-instr: AP = 52.23 (#pos = 285)\n",
      "              carry-obj: AP = 34.01 (#pos = 472)\n",
      "              throw-obj: AP = 44.48 (#pos = 244)\n",
      "              catch-obj: AP = 45.45 (#pos = 246)\n",
      "              cut-instr: AP = 34.09 (#pos = 269)\n",
      "                cut-obj: AP = 37.75 (#pos = 269)\n",
      " work_on_computer-instr: AP = 58.71 (#pos = 410)\n",
      "              ski-instr: AP = 46.44 (#pos = 424)\n",
      "             surf-instr: AP = 78.65 (#pos = 486)\n",
      "       skateboard-instr: AP = 81.91 (#pos = 417)\n",
      "            drink-instr: AP = 33.09 (#pos = 82)\n",
      "               kick-obj: AP = 66.52 (#pos = 180)\n",
      "               read-obj: AP = 23.66 (#pos = 111)\n",
      "        snowboard-instr: AP = 67.74 (#pos = 277)\n",
      "Average Role [scenario_1] AP = 45.74\n",
      "---------------------------------------------\n",
      "---------Reporting Role AP (%)------------------\n",
      "               hold-obj: AP = 39.20 (#pos = 3608)\n",
      "              sit-instr: AP = 25.56 (#pos = 1916)\n",
      "             ride-instr: AP = 64.97 (#pos = 556)\n",
      "               look-obj: AP = 37.58 (#pos = 3347)\n",
      "              hit-instr: AP = 71.49 (#pos = 349)\n",
      "                hit-obj: AP = 45.92 (#pos = 349)\n",
      "                eat-obj: AP = 40.11 (#pos = 521)\n",
      "              eat-instr: AP = 4.20 (#pos = 521)\n",
      "             jump-instr: AP = 48.05 (#pos = 635)\n",
      "              lay-instr: AP = 23.96 (#pos = 387)\n",
      "    talk_on_phone-instr: AP = 52.54 (#pos = 285)\n",
      "              carry-obj: AP = 34.39 (#pos = 472)\n",
      "              throw-obj: AP = 46.19 (#pos = 244)\n",
      "              catch-obj: AP = 46.68 (#pos = 246)\n",
      "              cut-instr: AP = 34.66 (#pos = 269)\n",
      "                cut-obj: AP = 39.09 (#pos = 269)\n",
      " work_on_computer-instr: AP = 60.03 (#pos = 410)\n",
      "              ski-instr: AP = 47.39 (#pos = 424)\n",
      "             surf-instr: AP = 79.18 (#pos = 486)\n",
      "       skateboard-instr: AP = 82.09 (#pos = 417)\n",
      "            drink-instr: AP = 34.13 (#pos = 82)\n",
      "               kick-obj: AP = 68.81 (#pos = 180)\n",
      "               read-obj: AP = 25.32 (#pos = 111)\n",
      "        snowboard-instr: AP = 68.14 (#pos = 277)\n",
      "Average Role [scenario_1] AP = 46.65\n",
      "---------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "det_name1 = '../Outputs/e2e_interact_net_R-50-FPN_1x/Dec10-15-24-10_wanbo_node23_step/test/hoi_vcoco_test_results.pkl'\n",
    "det_name2 = '../Outputs/e2e_interact_net_R-50-FPN_1x/Dec10-15-24-10_wanbo_node23_step/test/hoi_vcoco_test_results_no_misgrouping.pkl'\n",
    "dets1 = pickle.load(open(det_name1, 'rb') , encoding='latin1')\n",
    "dets2 = pickle.load(open(det_name2, 'rb') , encoding='latin1')\n",
    "json_dataset.VCOCO._do_role_eval(vcocodb, dets1, ovr_thresh=0.5, eval_type='scenario_1', plot_save_path=None)\n",
    "json_dataset.VCOCO._do_role_eval(vcocodb, dets2, ovr_thresh=0.5, eval_type='scenario_1', plot_save_path=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------Reporting Role AP (%)------------------\n",
      "               hold-obj: AP = 41.58 (#pos = 3608)\n",
      "              sit-instr: AP = 25.48 (#pos = 1916)\n",
      "             ride-instr: AP = 64.77 (#pos = 556)\n",
      "               look-obj: AP = 42.21 (#pos = 3347)\n",
      "              hit-instr: AP = 74.56 (#pos = 349)\n",
      "                hit-obj: AP = 47.55 (#pos = 349)\n",
      "                eat-obj: AP = 40.51 (#pos = 521)\n",
      "              eat-instr: AP = 5.07 (#pos = 521)\n",
      "             jump-instr: AP = 52.40 (#pos = 635)\n",
      "              lay-instr: AP = 28.97 (#pos = 387)\n",
      "    talk_on_phone-instr: AP = 54.59 (#pos = 285)\n",
      "              carry-obj: AP = 37.51 (#pos = 472)\n",
      "              throw-obj: AP = 62.98 (#pos = 244)\n",
      "              catch-obj: AP = 56.92 (#pos = 246)\n",
      "              cut-instr: AP = 36.05 (#pos = 269)\n",
      "                cut-obj: AP = 40.14 (#pos = 269)\n",
      " work_on_computer-instr: AP = 59.59 (#pos = 410)\n",
      "              ski-instr: AP = 48.20 (#pos = 424)\n",
      "             surf-instr: AP = 80.30 (#pos = 486)\n",
      "       skateboard-instr: AP = 83.64 (#pos = 417)\n",
      "            drink-instr: AP = 36.70 (#pos = 82)\n",
      "               kick-obj: AP = 78.19 (#pos = 180)\n",
      "               read-obj: AP = 24.95 (#pos = 111)\n",
      "        snowboard-instr: AP = 71.46 (#pos = 277)\n",
      "Average Role [scenario_1] AP = 49.76\n",
      "---------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "det_name3 = '../Outputs/e2e_interact_net_R-50-FPN_1x/Dec10-15-24-10_wanbo_node23_step/test/hoi_vcoco_test_results_no_actmiscls.pkl'\n",
    "dets3 = pickle.load(open(det_name3, 'rb') , encoding='latin1')\n",
    "json_dataset.VCOCO._do_role_eval(vcocodb, dets3, ovr_thresh=0.5, eval_type='scenario_1', plot_save_path=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------Reporting Role AP (%)------------------\n",
      "               hold-obj: AP = 42.99 (#pos = 3608)\n",
      "              sit-instr: AP = 29.53 (#pos = 1916)\n",
      "             ride-instr: AP = 66.14 (#pos = 556)\n",
      "               look-obj: AP = 39.26 (#pos = 3347)\n",
      "              hit-instr: AP = 72.15 (#pos = 349)\n",
      "                hit-obj: AP = 47.54 (#pos = 349)\n",
      "                eat-obj: AP = 43.16 (#pos = 521)\n",
      "              eat-instr: AP = 5.42 (#pos = 521)\n",
      "             jump-instr: AP = 48.74 (#pos = 635)\n",
      "              lay-instr: AP = 26.07 (#pos = 387)\n",
      "    talk_on_phone-instr: AP = 56.81 (#pos = 285)\n",
      "              carry-obj: AP = 40.13 (#pos = 472)\n",
      "              throw-obj: AP = 45.78 (#pos = 244)\n",
      "              catch-obj: AP = 46.49 (#pos = 246)\n",
      "              cut-instr: AP = 40.58 (#pos = 269)\n",
      "                cut-obj: AP = 42.35 (#pos = 269)\n",
      " work_on_computer-instr: AP = 62.85 (#pos = 410)\n",
      "              ski-instr: AP = 53.20 (#pos = 424)\n",
      "             surf-instr: AP = 80.18 (#pos = 486)\n",
      "       skateboard-instr: AP = 83.63 (#pos = 417)\n",
      "            drink-instr: AP = 40.29 (#pos = 82)\n",
      "               kick-obj: AP = 67.02 (#pos = 180)\n",
      "               read-obj: AP = 30.99 (#pos = 111)\n",
      "        snowboard-instr: AP = 71.07 (#pos = 277)\n",
      "Average Role [scenario_1] AP = 49.27\n",
      "---------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "det_name4 = '../Outputs/e2e_interact_net_R-50-FPN_1x/Dec10-15-24-10_wanbo_node23_step/test/hoi_vcoco_test_results_no_objmisloc.pkl'\n",
    "dets4 = pickle.load(open(det_name3, 'rb') , encoding='latin1')\n",
    "json_dataset.VCOCO._do_role_eval(vcocodb, dets4, ovr_thresh=0.5, eval_type='scenario_1', plot_save_path=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------Reporting Role AP (%)------------------\n",
      "               hold-obj: AP = 50.55 (#pos = 3608)\n",
      "              sit-instr: AP = 33.47 (#pos = 1916)\n",
      "             ride-instr: AP = 70.97 (#pos = 556)\n",
      "               look-obj: AP = 52.39 (#pos = 3347)\n",
      "              hit-instr: AP = 78.18 (#pos = 349)\n",
      "                hit-obj: AP = 54.49 (#pos = 349)\n",
      "                eat-obj: AP = 48.44 (#pos = 521)\n",
      "              eat-instr: AP = 8.87 (#pos = 521)\n",
      "             jump-instr: AP = 55.30 (#pos = 635)\n",
      "              lay-instr: AP = 36.89 (#pos = 387)\n",
      "    talk_on_phone-instr: AP = 60.61 (#pos = 285)\n",
      "              carry-obj: AP = 52.46 (#pos = 472)\n",
      "              throw-obj: AP = 72.78 (#pos = 244)\n",
      "              catch-obj: AP = 66.19 (#pos = 246)\n",
      "              cut-instr: AP = 47.82 (#pos = 269)\n",
      "                cut-obj: AP = 50.99 (#pos = 269)\n",
      " work_on_computer-instr: AP = 69.04 (#pos = 410)\n",
      "              ski-instr: AP = 60.16 (#pos = 424)\n",
      "             surf-instr: AP = 83.20 (#pos = 486)\n",
      "       skateboard-instr: AP = 86.20 (#pos = 417)\n",
      "            drink-instr: AP = 58.52 (#pos = 82)\n",
      "               kick-obj: AP = 84.34 (#pos = 180)\n",
      "               read-obj: AP = 45.94 (#pos = 111)\n",
      "        snowboard-instr: AP = 76.73 (#pos = 277)\n",
      "Average Role [scenario_1] AP = 58.52\n",
      "---------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# upper bound: 58.52\n",
    "det_name5 = '../Outputs/e2e_interact_net_R-50-FPN_1x/Dec10-15-24-10_wanbo_node23_step/test/hoi_vcoco_test_results_no_fp.pkl'\n",
    "dets5 = pickle.load(open(det_name5, 'rb') , encoding='latin1')\n",
    "json_dataset.VCOCO._do_role_eval(vcocodb, dets5, ovr_thresh=0.5, eval_type='scenario_1', plot_save_path=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------Reporting Role AP (%)------------------\n",
      "               hold-obj: AP = 42.48 (#pos = 3608)\n",
      "              sit-instr: AP = 26.30 (#pos = 1916)\n",
      "             ride-instr: AP = 66.28 (#pos = 556)\n",
      "               look-obj: AP = 44.49 (#pos = 3347)\n",
      "              hit-instr: AP = 75.36 (#pos = 349)\n",
      "                hit-obj: AP = 49.22 (#pos = 349)\n",
      "                eat-obj: AP = 41.73 (#pos = 521)\n",
      "              eat-instr: AP = 5.17 (#pos = 521)\n",
      "             jump-instr: AP = 52.83 (#pos = 635)\n",
      "              lay-instr: AP = 29.12 (#pos = 387)\n",
      "    talk_on_phone-instr: AP = 54.89 (#pos = 285)\n",
      "              carry-obj: AP = 38.09 (#pos = 472)\n",
      "              throw-obj: AP = 66.79 (#pos = 244)\n",
      "              catch-obj: AP = 59.62 (#pos = 246)\n",
      "              cut-instr: AP = 36.76 (#pos = 269)\n",
      "                cut-obj: AP = 41.72 (#pos = 269)\n",
      " work_on_computer-instr: AP = 61.01 (#pos = 410)\n",
      "              ski-instr: AP = 49.24 (#pos = 424)\n",
      "             surf-instr: AP = 80.89 (#pos = 486)\n",
      "       skateboard-instr: AP = 83.86 (#pos = 417)\n",
      "            drink-instr: AP = 38.29 (#pos = 82)\n",
      "               kick-obj: AP = 81.83 (#pos = 180)\n",
      "               read-obj: AP = 26.92 (#pos = 111)\n",
      "        snowboard-instr: AP = 72.02 (#pos = 277)\n",
      "Average Role [scenario_1] AP = 51.04\n",
      "---------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "det_name6 = '../Outputs/e2e_interact_net_R-50-FPN_1x/Dec10-15-24-10_wanbo_node23_step/test/hoi_vcoco_test_results_no_34error.pkl'\n",
    "dets6 = pickle.load(open(det_name6, 'rb') , encoding='latin1')\n",
    "json_dataset.VCOCO._do_role_eval(vcocodb, dets6, ovr_thresh=0.5, eval_type='scenario_1', plot_save_path=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
